import puppeteer from 'puppeteer'
import { writeFile } from 'fs'
import { ri } from './ri.js'
let browser;
let number = 0;
async function runT(arr) {
  // for (let i = 0; i < arr.length; i++) {
  const length = arr.length;
  let i = 0;
  while (i < length) {
    const elData = arr[i];


    if (elData.type === 'category' && elData.children && elData.children.length === 0 && elData.users === undefined) {
      try {
        if (number > 10) {
          number = 0;
          // browser.close();
          // browser = await puppeteer.launch({
          //   headless: 'new',
          // });
          const a = await new Promise((res) => {
            setTimeout(() => {
              res(5)
            }, 2000);
          });
          console.log('a----', a);
        }
        number++;
        const page = await browser.newPage();
        // await page.goto('https://www.baidu.com?word=' + elData.value, {
        //   timeout: 5 * 60 * 1000
        // });
        await page.goto('https://zh.wikipedia.org/wiki/Category:' + elData.value, {
          timeout: 5 * 60 * 1000
        })
        const arr = await page.evaluate(() => {
          var titlesList = document.querySelectorAll('.mw-category-group>ul>li>a');
          return Array.from(titlesList).map(el => el.innerText.trim()).join(',');
        });
        elData.users = arr;
        console.log('elData----', elData.users);
        page.close();
      } catch (error) {
        console.log('error---', error)
      }
    }

    if (elData.type === 'category' && elData.children && elData.children.length > 0) {
      runT(elData.children)
    }
    i++;
  }
  // }
}

async function scrape() {
  browser = await puppeteer.launch({
    headless: 'new',
  });
  const datas = [...ri];
  await runT(datas)


  writeFile("./netflixscrape.json", JSON.stringify(datas, null, 4), (err) => {
    if (err) {
      console.error(err);
      return;
    };
    console.log("Great Success");
  });
  browser.close()
}
scrape()
